{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ML_in_Finance-1D-CNNs\n",
    "# Author: Matthew Dixon\n",
    "# Version: 1.0 (24.7.2019)\n",
    "# License: MIT\n",
    "# Email: matthew.dixon@iit.edu\n",
    "# Notes: tested on Mac OS X with Python 3.6 and Tensorflow 1.3.0\n",
    "# Citation: Please cite the following reference if this notebook is used for research purposes:\n",
    "# Bilokon P., Dixon M.F. and I. Halperin, Machine Learning in Finance: From Theory to Practice, Springer Graduate textbook Series, 2020. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "\"\"\"\n",
    "Example of using Keras to implement a 1D convolutional neural network (CNN) for timeseries prediction.\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import print_function, division\n",
    "\n",
    "import numpy as np\n",
    "from keras.layers import Convolution1D, Dense, MaxPooling1D, Flatten\n",
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_timeseries_regressor(window_size, filter_length, nb_input_series=1, nb_outputs=1, nb_filter=4):\n",
    "    \"\"\":Return: a Keras Model for predicting the next value in a timeseries given a fixed-size lookback window of previous values.\n",
    "    The model can handle multiple input timeseries (`nb_input_series`) and multiple prediction targets (`nb_outputs`).\n",
    "    :param int window_size: The number of previous timeseries values to use as input features.  Also called lag or lookback.\n",
    "    :param int nb_input_series: The number of input timeseries; 1 for a single timeseries.\n",
    "      The `X` input to ``fit()`` should be an array of shape ``(n_instances, window_size, nb_input_series)``; each instance is\n",
    "      a 2D array of shape ``(window_size, nb_input_series)``.  For example, for `window_size` = 3 and `nb_input_series` = 1 (a\n",
    "      single timeseries), one instance could be ``[[0], [1], [2]]``. See ``make_timeseries_instances()``.\n",
    "    :param int nb_outputs: The output dimension, often equal to the number of inputs.\n",
    "      For each input instance (array with shape ``(window_size, nb_input_series)``), the output is a vector of size `nb_outputs`,\n",
    "      usually the value(s) predicted to come after the last value in that input instance, i.e., the next value\n",
    "      in the sequence. The `y` input to ``fit()`` should be an array of shape ``(n_instances, nb_outputs)``.\n",
    "    :param int filter_length: the size (along the `window_size` dimension) of the sliding window that gets convolved with\n",
    "      each position along each instance. The difference between 1D and 2D convolution is that a 1D filter's \"height\" is fixed\n",
    "      to the number of input timeseries (its \"width\" being `filter_length`), and it can only slide along the window\n",
    "      dimension.  This is useful as generally the input timeseries have no spatial/ordinal relationship, so it's not\n",
    "      meaningful to look for patterns that are invariant with respect to subsets of the timeseries.\n",
    "    :param int nb_filter: The number of different filters to learn (roughly, input patterns to recognize).\n",
    "    \"\"\"\n",
    "    model = Sequential((\n",
    "        # The first conv layer learns `nb_filter` filters (aka kernels), each of size ``(filter_length, nb_input_series)``.\n",
    "        # Its output will have shape (None, window_size - filter_length + 1, nb_filter), i.e., for each position in\n",
    "        # the input timeseries, the activation of each filter at that position.\n",
    "        Convolution1D(nb_filter=nb_filter, filter_length=filter_length, activation='relu', input_shape=(window_size, nb_input_series)),\n",
    "        #MaxPooling1D(),     # Downsample the output of convolution by 2X.\n",
    "        #Convolution1D(nb_filter=nb_filter, filter_length=filter_length, activation='relu'),\n",
    "        #MaxPooling1D(),\n",
    "        Flatten(),\n",
    "        Dense(nb_outputs, activation='linear'),     # For binary classification, change the activation to 'sigmoid'\n",
    "    ))\n",
    "    model.compile(loss='mse', optimizer='adam', metrics=['mae'])\n",
    "    # To perform (binary) classification instead:\n",
    "    # model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_timeseries_instances(timeseries, window_size):\n",
    "    \"\"\"Make input features and prediction targets from a `timeseries` for use in machine learning.\n",
    "    :return: A tuple of `(X, y, q)`.  `X` are the inputs to a predictor, a 3D ndarray with shape\n",
    "      ``(timeseries.shape[0] - window_size, window_size, timeseries.shape[1] or 1)``.  For each row of `X`, the\n",
    "      corresponding row of `y` is the next value in the timeseries.  The `q` or query is the last instance, what you would use\n",
    "      to predict a hypothetical next (unprovided) value in the `timeseries`.\n",
    "    :param ndarray timeseries: Either a simple vector, or a matrix of shape ``(timestep, series_num)``, i.e., time is axis 0 (the\n",
    "      row) and the series is axis 1 (the column).\n",
    "    :param int window_size: The number of samples to use as input prediction features (also called the lag or lookback).\n",
    "    \"\"\"\n",
    "    timeseries = np.asarray(timeseries)\n",
    "    assert 0 < window_size < timeseries.shape[0]\n",
    "    X = np.atleast_3d(np.array([timeseries[start:start + window_size] for start in range(0, timeseries.shape[0] - window_size)]))\n",
    "    y = timeseries[window_size:]\n",
    "    q = np.atleast_3d([timeseries[-window_size:]])\n",
    "    return X, y, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_timeseries(timeseries, window_size):\n",
    "    \"\"\"Create a 1D CNN regressor to predict the next value in a `timeseries` using the preceding `window_size` elements\n",
    "    as input features and evaluate its performance.\n",
    "    :param ndarray timeseries: Timeseries data with time increasing down the rows (the leading dimension/axis).\n",
    "    :param int window_size: The number of previous timeseries values to use to predict the next.\n",
    "    \"\"\"\n",
    "    filter_length = 5\n",
    "    nb_filter = 4\n",
    "    timeseries = np.atleast_2d(timeseries)\n",
    "    if timeseries.shape[0] == 1:\n",
    "        timeseries = timeseries.T       # Convert 1D vectors to 2D column vectors\n",
    "\n",
    "    nb_samples, nb_series = timeseries.shape\n",
    "    print('\\n\\nTimeseries ({} samples by {} series):\\n'.format(nb_samples, nb_series), timeseries)\n",
    "    model = make_timeseries_regressor(window_size=window_size, filter_length=filter_length, nb_input_series=nb_series, nb_outputs=nb_series, nb_filter=nb_filter)\n",
    "    print('\\n\\nModel with input size {}, output size {}, {} conv filters of length {}'.format(model.input_shape, model.output_shape, nb_filter, filter_length))\n",
    "    model.summary()\n",
    "\n",
    "    X, y, q = make_timeseries_instances(timeseries, window_size)\n",
    "    print('\\n\\nInput features:', X, '\\n\\nOutput labels:', y, '\\n\\nQuery vector:', q, sep='\\n')\n",
    "    test_size = int(0.01 * nb_samples)           # In real life you'd want to use 0.2 - 0.5\n",
    "    X_train, X_test, y_train, y_test = X[:-test_size], X[-test_size:], y[:-test_size], y[-test_size:]\n",
    "    model.fit(X_train, y_train, nb_epoch=25, batch_size=2, validation_data=(X_test, y_test))\n",
    "\n",
    "    pred = model.predict(X_test)\n",
    "    print('\\n\\nactual', 'predicted', sep='\\t')\n",
    "    for actual, predicted in zip(y_test, pred.squeeze()):\n",
    "        print(actual.squeeze(), predicted, sep='\\t')\n",
    "    print('next', model.predict(q).squeeze(), sep='\\t')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Simple single timeseries vector prediction\n",
      "\n",
      "\n",
      "Timeseries (1000 samples by 1 series):\n",
      " [[  0]\n",
      " [  1]\n",
      " [  2]\n",
      " ...\n",
      " [997]\n",
      " [998]\n",
      " [999]]\n",
      "\n",
      "\n",
      "Model with input size (None, 50, 1), output size (None, 1), 4 conv filters of length 5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_5 (Conv1D)            (None, 46, 4)             24        \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 184)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 185       \n",
      "=================================================================\n",
      "Total params: 209\n",
      "Trainable params: 209\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "\n",
      "Input features:\n",
      "[[[  0]\n",
      "  [  1]\n",
      "  [  2]\n",
      "  ...\n",
      "  [ 47]\n",
      "  [ 48]\n",
      "  [ 49]]\n",
      "\n",
      " [[  1]\n",
      "  [  2]\n",
      "  [  3]\n",
      "  ...\n",
      "  [ 48]\n",
      "  [ 49]\n",
      "  [ 50]]\n",
      "\n",
      " [[  2]\n",
      "  [  3]\n",
      "  [  4]\n",
      "  ...\n",
      "  [ 49]\n",
      "  [ 50]\n",
      "  [ 51]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[947]\n",
      "  [948]\n",
      "  [949]\n",
      "  ...\n",
      "  [994]\n",
      "  [995]\n",
      "  [996]]\n",
      "\n",
      " [[948]\n",
      "  [949]\n",
      "  [950]\n",
      "  ...\n",
      "  [995]\n",
      "  [996]\n",
      "  [997]]\n",
      "\n",
      " [[949]\n",
      "  [950]\n",
      "  [951]\n",
      "  ...\n",
      "  [996]\n",
      "  [997]\n",
      "  [998]]]\n",
      "\n",
      "\n",
      "Output labels:\n",
      "[[ 50]\n",
      " [ 51]\n",
      " [ 52]\n",
      " ...\n",
      " [997]\n",
      " [998]\n",
      " [999]]\n",
      "\n",
      "\n",
      "Query vector:\n",
      "[[[950]\n",
      "  [951]\n",
      "  [952]\n",
      "  ...\n",
      "  [997]\n",
      "  [998]\n",
      "  [999]]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/matthewdixon/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:24: UserWarning: Update your `Conv1D` call to the Keras 2 API: `Conv1D(activation=\"relu\", input_shape=(50, 1), filters=4, kernel_size=5)`\n",
      "/Users/matthewdixon/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:23: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 940 samples, validate on 10 samples\n",
      "Epoch 1/25\n",
      "940/940 [==============================] - 1s 767us/step - loss: 12798.0792 - mean_absolute_error: 31.7979 - val_loss: 414.0209 - val_mean_absolute_error: 20.3471\n",
      "Epoch 2/25\n",
      "940/940 [==============================] - 0s 496us/step - loss: 105.5716 - mean_absolute_error: 8.6100 - val_loss: 23.9553 - val_mean_absolute_error: 4.8939\n",
      "Epoch 3/25\n",
      "940/940 [==============================] - 0s 486us/step - loss: 98.9789 - mean_absolute_error: 8.3387 - val_loss: 0.0547 - val_mean_absolute_error: 0.2272\n",
      "Epoch 4/25\n",
      "940/940 [==============================] - 1s 727us/step - loss: 92.6017 - mean_absolute_error: 8.1051 - val_loss: 72.7340 - val_mean_absolute_error: 8.5281\n",
      "Epoch 5/25\n",
      "940/940 [==============================] - 1s 581us/step - loss: 84.0197 - mean_absolute_error: 7.8058 - val_loss: 2.3349 - val_mean_absolute_error: 1.5271\n",
      "Epoch 6/25\n",
      "940/940 [==============================] - 0s 439us/step - loss: 74.3767 - mean_absolute_error: 7.2539 - val_loss: 137.3759 - val_mean_absolute_error: 11.7205\n",
      "Epoch 7/25\n",
      "940/940 [==============================] - 0s 499us/step - loss: 66.0212 - mean_absolute_error: 6.7875 - val_loss: 4.1289 - val_mean_absolute_error: 2.0315\n",
      "Epoch 8/25\n",
      "940/940 [==============================] - 0s 490us/step - loss: 89.1669 - mean_absolute_error: 7.0634 - val_loss: 3.1426 - val_mean_absolute_error: 1.7724\n",
      "Epoch 9/25\n",
      "940/940 [==============================] - 1s 638us/step - loss: 41.6476 - mean_absolute_error: 5.2347 - val_loss: 36.9568 - val_mean_absolute_error: 6.0791\n",
      "Epoch 10/25\n",
      "940/940 [==============================] - 0s 453us/step - loss: 34.7847 - mean_absolute_error: 4.8046 - val_loss: 21.3841 - val_mean_absolute_error: 4.6242\n",
      "Epoch 11/25\n",
      "940/940 [==============================] - 0s 435us/step - loss: 27.7978 - mean_absolute_error: 4.2096 - val_loss: 20.1202 - val_mean_absolute_error: 4.4855\n",
      "Epoch 12/25\n",
      "940/940 [==============================] - 0s 439us/step - loss: 38.9460 - mean_absolute_error: 4.4142 - val_loss: 47.7895 - val_mean_absolute_error: 6.9130\n",
      "Epoch 13/25\n",
      "940/940 [==============================] - 0s 431us/step - loss: 30.0796 - mean_absolute_error: 3.5847 - val_loss: 5.1933 - val_mean_absolute_error: 2.2788\n",
      "Epoch 14/25\n",
      "940/940 [==============================] - 0s 448us/step - loss: 21.8589 - mean_absolute_error: 3.2256 - val_loss: 5.5032 - val_mean_absolute_error: 2.3458\n",
      "Epoch 15/25\n",
      "940/940 [==============================] - 0s 444us/step - loss: 25.4800 - mean_absolute_error: 2.9572 - val_loss: 5.0175 - val_mean_absolute_error: 2.2399\n",
      "Epoch 16/25\n",
      "940/940 [==============================] - 0s 464us/step - loss: 438.0670 - mean_absolute_error: 7.4128 - val_loss: 0.8200 - val_mean_absolute_error: 0.9055\n",
      "Epoch 17/25\n",
      "940/940 [==============================] - 0s 447us/step - loss: 3.0146 - mean_absolute_error: 1.4793 - val_loss: 0.1239 - val_mean_absolute_error: 0.3519\n",
      "Epoch 18/25\n",
      "940/940 [==============================] - 0s 443us/step - loss: 1.9695 - mean_absolute_error: 1.1828 - val_loss: 1.5294 - val_mean_absolute_error: 1.2366\n",
      "Epoch 19/25\n",
      "940/940 [==============================] - 0s 441us/step - loss: 3.2093 - mean_absolute_error: 1.3065 - val_loss: 0.0897 - val_mean_absolute_error: 0.2995\n",
      "Epoch 20/25\n",
      "940/940 [==============================] - 0s 450us/step - loss: 1.6602 - mean_absolute_error: 1.0171 - val_loss: 0.3694 - val_mean_absolute_error: 0.6078\n",
      "Epoch 21/25\n",
      "940/940 [==============================] - 0s 446us/step - loss: 82.2746 - mean_absolute_error: 4.8090 - val_loss: 5.5217 - val_mean_absolute_error: 2.3498\n",
      "Epoch 22/25\n",
      "940/940 [==============================] - 0s 444us/step - loss: 2.9527 - mean_absolute_error: 1.1639 - val_loss: 0.3320 - val_mean_absolute_error: 0.5762\n",
      "Epoch 23/25\n",
      "940/940 [==============================] - 0s 439us/step - loss: 1.2681 - mean_absolute_error: 0.8377 - val_loss: 8.4245 - val_mean_absolute_error: 2.9025\n",
      "Epoch 24/25\n",
      "940/940 [==============================] - 0s 447us/step - loss: 24.9942 - mean_absolute_error: 2.3324 - val_loss: 0.6685 - val_mean_absolute_error: 0.8176\n",
      "Epoch 25/25\n",
      "940/940 [==============================] - 0s 446us/step - loss: 0.7619 - mean_absolute_error: 0.5759 - val_loss: 0.0126 - val_mean_absolute_error: 0.1123\n",
      "\n",
      "\n",
      "actual\tpredicted\n",
      "990\t989.88617\n",
      "991\t990.8865\n",
      "992\t991.8869\n",
      "993\t992.8873\n",
      "994\t993.8876\n",
      "995\t994.888\n",
      "996\t995.8882\n",
      "997\t996.88855\n",
      "998\t997.8889\n",
      "999\t998.8893\n",
      "next\t999.88947\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Prepare input data, build model, evaluate.\"\"\"\n",
    "np.set_printoptions(threshold=25)\n",
    "ts_length = 1000\n",
    "window_size = 50\n",
    "\n",
    "print('\\nSimple single timeseries vector prediction')\n",
    "timeseries = np.arange(ts_length)                   # The timeseries f(t) = t\n",
    "model=evaluate_timeseries(timeseries, window_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights=model.get_layer('conv1d_3').weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'conv1d_3/kernel:0' shape=(5, 1, 4) dtype=float32_ref>,\n",
       " <tf.Variable 'conv1d_3/bias:0' shape=(4,) dtype=float32_ref>]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('\\nMultiple-input, multiple-output prediction')\n",
    "timeseries = np.array([np.arange(ts_length), -np.arange(ts_length)]).T      # The timeseries f(t) = [t, -t]\n",
    "evaluate_timeseries(timeseries, window_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
